{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# The code performs logistic/softmax regression on the MNIST Dataset\n",
    "# It is based upon the tutorial given at Tensorflow.org : https://www.tensorflow.org/get_started/mnist/beginners\n",
    "# We have added few Tensorboard summaries to understand it better\n",
    "\n",
    "# Import Modules needed\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt,  matplotlib.image as mpimg\n",
    "%matplotlib inline\n",
    "\n",
    "# Import Input \n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define Hyper Parameters of the model:\n",
    "learning_rate = 0.05\n",
    "batch_size = 100\n",
    "max_epochs = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x23cb5a81518>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEvZJREFUeJzt3X/wVXWdx/HnqxTdwF9gi0gWohCbW6CR48yi6VquMeMY\nGk5YhuGI/dg2a2daoxzIzTS32mH7geHIpiRWg5VkP7SMoh23RnSIQJHMcOU3Zi4/inWl9/5xD+6V\nvvdzv99z7q8vn9dj5jvfe8/7ns95c7iv7zn3nHvvUURgZvl5SbcbMLPucPjNMuXwm2XK4TfLlMNv\nlimH3yxTDr9Zpno2/JJC0h5J13e7F7NOk/QVSX+UtLFdy+jZ8BcmRsTHGhUlnStpnaQ/SFou6VVl\nFyRpkqSHirEekjSpwlhjin7+UPT3pgpjDZf0reIP4ZOSLq0w1mGSFknaKWmrpA9XGEuSPi3pd8XP\npyWpwngfKnraWfR4WIWxLi3W1R5J35Y0vOQ4QyQtlbSh2BidXbanYrx+P8ci4nLgLVWW11RE9OQP\nEMDJifqxwH8D04HDgX8Bfl5yWUOAJ4EPAYcB/1DcH1JyvP8EPgf8BXAx8Czw8pJj3Ql8HRgGTCn+\nzaeUHOsG4GfAMcBfAVuB80uOdRXwGPAKYDTwCPCekmP9HbANOKXo7SfAjSXHOgXYBZxVrLMlwNcq\nPC+uLtb7FuDsCs/nAT/HgLOBjWWX2bSndg1cubHm4Z8NPFB3fyjwR2BCiWWdB2wCVDftv8oEAxgP\n/A9wRN20FWWCUfybngPG1027vUIwNgPn1d2/rkIwHgBm192fVeGP7xLgU3X3/xbYWnKsTwFL6u6f\nVKzDI8qMVzfOxorhH/BzrN3h7/Xd/pRTgF/uvxMRe4DHi+llxlodxRov/LLCWE9ExK4WjDUeeD4i\n1lcdS9IxwCjq1lmFvuCA9d+GsUZKGlF1rIj4DbU/xuNL9tYqrXyOtcRgDv8warvA9XYCRxxkY+1s\n4Vjw4t7KjrV/vAPHGlbydX9fY1Gyt1au/1bqub4Gc/h3A0ceMO0oaq/3PFbfY3HAeGXH2j/egWPt\nPmDLVmUsSvbWynXWSj3X12AO/1pg4v47koZSe323tuRYrztgq/W6CmONlVT/F31iybHWA4dIGld1\nrIj4PbWDVhPrJpftCw5Y/20Ya1tE/K7qWJJOonawbX3DOTqjlc+x1mjXwYSqPzQ/4PdyartRF1M7\n2n8T1Y/2f5DWHO3/OfCZoq+LqHa0/2vUjvgPpfrR/huBn9Kao/3vAR6ldqS/6tH+84teXkNrjvbv\nBM4s1lnpo/3FeIcV/48bqR20O5y6g3btfI7ho/3Jx7wJWEftKP9PgDF1tTnA9wewvFOBh4qxHgZO\nrau9A1g7gLHGFP38kdrpsDfV1c6ktnvc37GGA98G9lA7OnxpXe2V1HYnX9nPsQ4DFhXh2AZ8+ID6\nbuDMfo6l4g/uM8XPTbz4SPZa4B0D+Hd+uOhpJ/DvwGF1te8DcwYw1qXFutoD3A0Mr6vdDNw8gLE2\nFM/F+p8xnXiOtTv8KhbScyTtpXaU9t8i4tpu92PWSZJupfYelu0RcXJbltGr4Tez9hrMB/zMrAKH\n3yxTh3RyYZL8GsOszSKiX2+0qrTll3S+pMckPS7pmipjmVlnlT7gJ+ml1N448WZq50AfBGZExCOJ\nebzlN2uzTmz5Twcej4gnIuI5am9GubDCeGbWQVXCPxp4qu7+xmLai0iaLWmlpJUVlmVmLdb2A34R\nsRBYCN7tN+slVbb8m4AT6u6/ophmZoNAlfA/CIyTdKKkIcDbgWWtacvM2q30bn9EPC/p74F7gZcC\niyKiex9PNLMB6eh7+/2a36z9OvImHzMbvBx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTDr9Z\nphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlMNv\nlimH3yxTDr9Zphx+s0w5/GaZcvjNMuXwm2Wq9CW67eBw8sknJ+uf//znk/Xx48cn62PHjm1Y27Fj\nR3Lee++9N1kfM2ZMsr58+fKGtS9/+cvJeTdt2pSsHwwqhV/SBmAXsA94PiImt6IpM2u/Vmz5z4mI\np1swjpl1kF/zm2WqavgD+JGkhyTN7usBkmZLWilpZcVlmVkLVd3tnxIRmyT9JfBDSesiYkX9AyJi\nIbAQQFJUXJ6ZtUilLX9EbCp+bwe+BZzeiqbMrP1Kh1/SUElH7L8NnAesaVVjZtZeiii3Jy5pLLWt\nPdRePiyJiOubzOPd/jZInaufM2dOct7p06cn6y972ctK9bSfpIa1ss+9VtizZ0+yPn/+/GT9+uuT\nT3X27t074J5aJSIar/Q6pV/zR8QTwMSy85tZd/lUn1mmHH6zTDn8Zply+M0y5fCbZar0qb5SC/Op\nvlJGjx6drH/3u99tWHvta1+bnPe5555L1u+7775k/a677krWH3jggWQ9ZdiwYcn6xRdfnKzPmjWr\nYW3UqFHJeZvlYs2a9Ftapk6dmqy38yPD/T3V5y2/WaYcfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5Yp\nn+fvAVOmTEnWly1blqwfffTRDWu7d+9OzvuRj3wkWb/55puT9V527LHHNqx96UtfSs47bdq0ZP0l\nL0lvNy+//PJkffHixcl6FT7Pb2ZJDr9Zphx+s0w5/GaZcvjNMuXwm2XK4TfLlM/zd8CRRx6ZrK9c\nmb6S2UknnZSsb968uWHtiiuuSM7b7PP6B6tzzjknWZ8wYUKy/oUvfCFZb3Zp86uvvjpZr8Ln+c0s\nyeE3y5TDb5Yph98sUw6/WaYcfrNMOfxmmfJ5/g5IXUIbYP369ZXGP/fccxvWli9fXmnsXnbBBRck\n63fffXfDWurS4QDjxo1L1ptdunz16tXJeju17Dy/pEWStktaUzdtuKQfSvp18fuYKs2aWef1Z7f/\nK8D5B0y7Brg/IsYB9xf3zWwQaRr+iFgBPHPA5AuB24rbtwFvbXFfZtZmh5Scb2REbClubwVGNnqg\npNnA7JLLMbM2KRv+F0REpA7kRcRCYCHke8DPrBeVPdW3TdIogOL39ta1ZGadUDb8y4CZxe2ZQONz\nKmbWk5ru9ku6EzgbOFbSRmAucCPwDUlXAE8Cl7SzyYNd1fdajBzZ8JALxx13XHLerVu3Vlp2M6lr\nCrz61a9OznvppZcm67NmzUrWU+t1xYoVyXk3btyYrO/duzdZHwyahj8iZjQoNX5niZn1PL+91yxT\nDr9Zphx+s0w5/GaZcvjNMuWP9HbAiBEjkvVmp52afY106uOp27en33+1YMGCZP2OO+5I1t/97ncn\n6+9617sa1o4//vjkvM3s27cvWf/xj3/csDZ9+vTkvLt27SrVUy/wV3ebWZLDb5Yph98sUw6/WaYc\nfrNMOfxmmXL4zTLl8/w9oNnHbmfOnJmsz5s3r2FtyJAhZVpqmdR7EJo995566qlk/aKLLkrWH374\n4WT9YOXz/GaW5PCbZcrhN8uUw2+WKYffLFMOv1mmHH6zTPk8/0Fg8eLFDWvNvv663aqc52/2teIT\nJ05M1p9++ulk/WDl8/xmluTwm2XK4TfLlMNvlimH3yxTDr9Zphx+s0z5PP8g8MUvfjFZf9/73tew\n1uz/95577knWr7vuumR95cqVyfo73/nOhrXTTjstOe+VV16ZrK9bty5Zf8Mb3pCsH6xadp5f0iJJ\n2yWtqZs2T9ImSauKn6lVmjWzzuvPbv9XgPP7mP6vETGp+Plea9sys3ZrGv6IWAE804FezKyDqhzw\n+4Ck1cXLgmMaPUjSbEkrJaVfHJpZR5UN/wJgLDAJ2AJ8ttEDI2JhREyOiMkll2VmbVAq/BGxLSL2\nRcSfgFuA01vblpm1W6nwSxpVd3casKbRY82sNzU9zy/pTuBs4FhgGzC3uD8JCGADcFVEbGm6MJ/n\n79NHP/rRZH3u3LnJeuq7+T/5yU8m5/3EJz6RrO/bty9Zb6dp06Yl67feemuyPnbs2Ia1Z599tlRP\ng0F/z/Mf0o+BZvQxOb3Wzazn+e29Zply+M0y5fCbZcrhN8uUw2+WqaZH+626GTP6OmHy/z7+8Y8n\n64ceemiyftVVVzWsLVq0KDlvN0/lVXXUUUcl62eccUbD2g9+8INWtzPoeMtvlimH3yxTDr9Zphx+\ns0w5/GaZcvjNMuXwm2XK5/k7YN68ecn64YcfnqzPnz8/Wb/lllsG2lJPOO6445L1Zh9HbubEE0+s\nNP/Bzlt+s0w5/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxTPs/fAhMmTEjWjz/++GR9165dyfoNN9ww\n4J4Ggze+8Y3JerP12my9LV26dMA95cRbfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5Yph98sU03P80s6\nAbgdGEntktwLI2K+pOHA14Ex1C7TfUlE/L59rfau9773vcn60KFDk/Vm3+u/Y8eOAffUK6699tqG\ntWaXB9+zZ0+yvmDBgmR9MK+3TujPlv954B8j4jXAGcD7Jb0GuAa4PyLGAfcX981skGga/ojYEhEP\nF7d3AY8Co4ELgduKh90GvLVdTZpZ6w3oNb+kMcCpwC+AkRGxpShtpfaywMwGiX6/t1/SMOAu4OqI\n2CnphVpEhKRoMN9sYHbVRs2stfq15Zd0KLXg3xER3ywmb5M0qqiPArb3NW9ELIyIyRExuRUNm1lr\nNA2/apv4W4FHI+JzdaVlwMzi9kzg7ta3Z2bt0p/d/r8BLgN+JWlVMW0OcCPwDUlXAE8Cl7SnxcEv\nos9XRC/47W9/26FOBu60005L1ufOnZusX3DBBQ1rzdbLmWeemayvWrUqWbe0puGPiP8A1KB8bmvb\nMbNO8Tv8zDLl8JtlyuE3y5TDb5Yph98sUw6/Wab81d094Ktf/WqyvmTJkmR98+bNDWsjRoxIzvu2\nt70tWW/29dnNLi+e+ljuokWLkvOuXbs2WbdqvOU3y5TDb5Yph98sUw6/WaYcfrNMOfxmmXL4zTKl\nZp+pbunCGnzV12D3+te/Plm/7777kvWjjz660vLrv1LtQFX/f/fu3ZusNzsXf9lllzWsPfbYY6V6\nsrSIaPyEqOMtv1mmHH6zTDn8Zply+M0y5fCbZcrhN8uUw2+WKZ/n74CzzjorWZ82bVqy3uwz96NH\nj25Ye/DBB5PzLl26NFn/zne+k6yvW7cuWbfO83l+M0ty+M0y5fCbZcrhN8uUw2+WKYffLFMOv1mm\nmp7nl3QCcDswEghgYUTMlzQPuBLYUTx0TkR8r8lYWZ7nN+uk/p7n70/4RwGjIuJhSUcADwFvBS4B\ndkfEZ/rblMNv1n79DX/TK/ZExBZgS3F7l6RHgcZvKTOzQWFAr/kljQFOBX5RTPqApNWSFkk6psE8\nsyWtlLSyUqdm1lL9fm+/pGHAT4HrI+KbkkYCT1M7DvDP1F4azGoyhnf7zdqsZa/5ASQdCtwD3BsR\nn+ujPga4JyL+usk4Dr9Zm7Xsgz2qfTXsrcCj9cEvDgTuNw1YM9Amzax7+nO0fwrwM+BXwJ+KyXOA\nGcAkarv9G4CrioODqbG85Tdrs5bu9reKw2/Wfv48v5klOfxmmXL4zTLl8JtlyuE3y5TDb5Yph98s\nUw6/WaYcfrNMOfxmmXL4zTLl8JtlyuE3y5TDb5appl/g2WJPA0/W3T+2mNaLerW3Xu0L3FtZrezt\nVf19YEc/z/9nC5dWRsTkrjWQ0Ku99Wpf4N7K6lZv3u03y5TDb5apbod/YZeXn9KrvfVqX+DeyupK\nb119zW9m3dPtLb+ZdYnDb5aproRf0vmSHpP0uKRrutFDI5I2SPqVpFXdvr5gcQ3E7ZLW1E0bLumH\nkn5d/O7zGold6m2epE3FulslaWqXejtB0nJJj0haK+mDxfSurrtEX11Zbx1/zS/ppcB64M3ARuBB\nYEZEPNLRRhqQtAGYHBFdf0OIpLOA3cDt+y+FJukm4JmIuLH4w3lMRPxTj/Q2jwFetr1NvTW6rPzl\ndHHdtfJy963QjS3/6cDjEfFERDwHfA24sAt99LyIWAE8c8DkC4Hbitu3UXvydFyD3npCRGyJiIeL\n27uA/ZeV7+q6S/TVFd0I/2jgqbr7G+niCuhDAD+S9JCk2d1upg8j6y6LthUY2c1m+tD0su2ddMBl\n5Xtm3ZW53H2r+YDfn5sSEZOAtwDvL3Zve1LUXrP10rnaBcBYatdw3AJ8tpvNFJeVvwu4OiJ21te6\nue766Ksr660b4d8EnFB3/xXFtJ4QEZuK39uBb1F7mdJLtu2/QnLxe3uX+3lBRGyLiH0R8SfgFrq4\n7orLyt8F3BER3ywmd33d9dVXt9ZbN8L/IDBO0omShgBvB5Z1oY8/I2locSAGSUOB8+i9S48vA2YW\nt2cCd3exlxfplcu2N7qsPF1edz13ufuI6PgPMJXaEf/fAB/rRg8N+hoL/LL4Wdvt3oA7qe0G/i+1\nYyNXACOA+4FfAz8ChvdQb4upXcp9NbWgjepSb1Oo7dKvBlYVP1O7ve4SfXVlvfntvWaZ8gE/s0w5\n/GaZcvjNMuXwm2XK4TfLlMNvlimH3yxT/wfZEe9DJysTLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x23cb56b9080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the data\n",
    "i = 55\n",
    "img = mnist.train.images[i]\n",
    "img=img.reshape((28,28))\n",
    "plt.imshow(img,cmap='gray')\n",
    "plt.title(mnist.train.labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Tensorflow graph inputs\n",
    "x = tf.placeholder(tf.float32, [None, 784], name='X')\n",
    "y = tf.placeholder(tf.float32, [None, 10],name='Y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning Variables\n",
    "W = tf.Variable(tf.zeros([784, 10]), name='W')\n",
    "b = tf.Variable(tf.zeros([10]), name='b')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the model\n",
    "with tf.name_scope(\"wx_b\") as scope:\n",
    "    y_hat = tf.nn.softmax(tf.matmul(x,W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Add summary ops to collect data while training\n",
    "w_h = tf.summary.histogram(\"weights\", W)\n",
    "b_h = tf.summary.histogram(\"biases\", b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define the cross-entropy loss function\n",
    "with tf.name_scope('cross-entropy') as scope:\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_hat))\n",
    "    tf.summary.scalar('cross-entropy', loss)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Choose the optimizer\n",
    "with tf.name_scope('Train') as scope:\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.01).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define ops to test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y_hat, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Merge All summaries\n",
    "merged_summary_op = tf.summary.merge_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: Loss 2.250637441548434\n",
      "Epoch 1: Loss 2.0909123147617685\n",
      "Epoch 2: Loss 1.961304726817391\n",
      "Epoch 3: Loss 1.8826904947107488\n",
      "Epoch 4: Loss 1.8332543024149808\n",
      "Epoch 5: Loss 1.8014142487265847\n",
      "Epoch 6: Loss 1.7794645149057562\n",
      "Epoch 7: Loss 1.7633150293610313\n",
      "Epoch 8: Loss 1.750868919545954\n",
      "Epoch 9: Loss 1.7409239610758696\n",
      "Epoch 10: Loss 1.7327635704387319\n",
      "Epoch 11: Loss 1.7259209247068925\n",
      "Epoch 12: Loss 1.7200765184922653\n",
      "Epoch 13: Loss 1.7150074254382741\n",
      "Epoch 14: Loss 1.7105629732392051\n",
      "Epoch 15: Loss 1.7066086411476136\n",
      "Epoch 16: Loss 1.7030719780921937\n",
      "Epoch 17: Loss 1.6998750695315274\n",
      "Epoch 18: Loss 1.6969586805863814\n",
      "Epoch 19: Loss 1.6942723577672785\n",
      "Epoch 20: Loss 1.6917831217158925\n",
      "Epoch 21: Loss 1.6894430344755\n",
      "Epoch 22: Loss 1.6871924673427234\n",
      "Epoch 23: Loss 1.6849619973789562\n",
      "Epoch 24: Loss 1.6825617228854786\n",
      "Epoch 25: Loss 1.679472615068609\n",
      "Epoch 26: Loss 1.6742813853784042\n",
      "Epoch 27: Loss 1.6670515164462003\n",
      "Epoch 28: Loss 1.6613866899230263\n",
      "Epoch 29: Loss 1.6567579026655717\n",
      "Epoch 30: Loss 1.6523656353083525\n",
      "Epoch 31: Loss 1.648194577260451\n",
      "Epoch 32: Loss 1.6442665188962764\n",
      "Epoch 33: Loss 1.6406090131672946\n",
      "Epoch 34: Loss 1.6372782507809727\n",
      "Epoch 35: Loss 1.6342750826748935\n",
      "Epoch 36: Loss 1.6316023343259638\n",
      "Epoch 37: Loss 1.6292163107611917\n",
      "Epoch 38: Loss 1.627083955894817\n",
      "Epoch 39: Loss 1.625151891708374\n",
      "Epoch 40: Loss 1.623395596634258\n",
      "Epoch 41: Loss 1.6217725591226058\n",
      "Epoch 42: Loss 1.6202726106210188\n",
      "Epoch 43: Loss 1.6188621009479869\n",
      "Epoch 44: Loss 1.61754553491419\n",
      "Epoch 45: Loss 1.6162967278740623\n",
      "Epoch 46: Loss 1.6151180765845559\n",
      "Epoch 47: Loss 1.6139911796829918\n",
      "Epoch 48: Loss 1.6129282929680564\n",
      "Epoch 49: Loss 1.6119044132666154\n",
      "Epoch 50: Loss 1.6109310403737156\n",
      "Epoch 51: Loss 1.6099914607134733\n",
      "Epoch 52: Loss 1.609090700149536\n",
      "Epoch 53: Loss 1.60823002836921\n",
      "Epoch 54: Loss 1.6073956309665334\n",
      "Epoch 55: Loss 1.606584829200398\n",
      "Epoch 56: Loss 1.605816740989685\n",
      "Epoch 57: Loss 1.6050650377707048\n",
      "Epoch 58: Loss 1.6043416181477634\n",
      "Epoch 59: Loss 1.6036366787823764\n",
      "Epoch 60: Loss 1.602958100492304\n",
      "Epoch 61: Loss 1.6022983934662558\n",
      "Epoch 62: Loss 1.601654678258029\n",
      "Epoch 63: Loss 1.6010403561592101\n",
      "Epoch 64: Loss 1.6004306676171043\n",
      "Epoch 65: Loss 1.5998455500602722\n",
      "Epoch 66: Loss 1.5992759305780584\n",
      "Epoch 67: Loss 1.5987146282196045\n",
      "Epoch 68: Loss 1.5981726184758274\n",
      "Epoch 69: Loss 1.5976459035006436\n",
      "Epoch 70: Loss 1.5971321754022079\n",
      "Epoch 71: Loss 1.5966263669187373\n",
      "Epoch 72: Loss 1.59613530700857\n",
      "Epoch 73: Loss 1.5956569853695957\n",
      "Epoch 74: Loss 1.595184608372775\n",
      "Epoch 75: Loss 1.5947283653779463\n",
      "Epoch 76: Loss 1.5942823336341165\n",
      "Epoch 77: Loss 1.5938406662507492\n",
      "Epoch 78: Loss 1.5934169320626692\n",
      "Epoch 79: Loss 1.592994540387934\n",
      "Epoch 80: Loss 1.592581994750283\n",
      "Epoch 81: Loss 1.5921791280399669\n",
      "Epoch 82: Loss 1.5917877294800498\n",
      "Epoch 83: Loss 1.5913979604027488\n",
      "Epoch 84: Loss 1.5910226277871566\n",
      "Epoch 85: Loss 1.5906475175510753\n",
      "Epoch 86: Loss 1.5902841624346646\n",
      "Epoch 87: Loss 1.5899248957633971\n",
      "Epoch 88: Loss 1.58957210150632\n",
      "Epoch 89: Loss 1.589228289344094\n",
      "Epoch 90: Loss 1.5888881715861234\n",
      "Epoch 91: Loss 1.5885552321780811\n",
      "Epoch 92: Loss 1.5882277859341014\n",
      "Epoch 93: Loss 1.5879071083935825\n",
      "Epoch 94: Loss 1.5875906484777278\n",
      "Epoch 95: Loss 1.5872807639295405\n",
      "Epoch 96: Loss 1.5869747601855886\n",
      "Epoch 97: Loss 1.5866698317094283\n",
      "Epoch 98: Loss 1.5863799944790926\n",
      "Epoch 99: Loss 1.5860854231227528\n",
      "Done\n",
      "0.9091\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)  # initialize all variables\n",
    "    summary_writer = tf.summary.FileWriter('graphs', sess.graph)  # Create an event file\n",
    "    \n",
    "    # Training\n",
    "    for epoch in range(max_epochs):\n",
    "        loss_avg = 0\n",
    "        num_of_batch = int(mnist.train.num_examples/batch_size)\n",
    "        for i in range(num_of_batch):\n",
    "            batch_xs, batch_ys = mnist.train.next_batch(100)  # get the next batch of data\n",
    "            _, l, summary_str = sess.run([optimizer,loss, merged_summary_op], feed_dict={x: batch_xs, y: batch_ys})  # Run the optimizer\n",
    "            loss_avg += l\n",
    "            summary_writer.add_summary(summary_str, epoch*num_of_batch + i)  # Add all summaries per batch\n",
    "            \n",
    "        loss_avg = loss_avg/num_of_batch\n",
    "        print('Epoch {0}: Loss {1}'.format(epoch, loss_avg))\n",
    "    \n",
    "    print('Done')\n",
    "    \n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,y: mnist.test.labels}))\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
